#!/usr/bin/python
import json
from collections import OrderedDict
import os
import shutil

def addAll(path, contains_weixin=True):
    all = list()
    urls = set()
    for i in os.listdir(path):
        fp = os.path.join(path,i)
        print(fp)
        if (os.path.isdir(fp)):
            if fp[-6:] != 'weixin' or contains_weixin:
                all.extend(addAll(fp, contains_weixin))
            else:
                print('ignore weixin directory')
        else:
            if fp[-3:] == 'was':
                with open(fp,"r", encoding='utf-8') as f:
                    content = f.read()
                    temp_json = json.loads(content, object_pairs_hook=OrderedDict)
                    if type(temp_json) == list:
                        all.extend(temp_json)
                    else:
                        all.append(temp_json)
    return removeDup(all)
                

def checkDup():
    with open('all/builtin/base.was','r') as f:
        content = f.read()
        base = json.loads(content)
        urls = set()
        for each in base:
            urls.add(each['siteHostUrl'])

        path = 'all/builtin/novel'
        for i in os.listdir(path):
            fp = os.path.join(path,i)
            with open(fp,'r') as n:
                novel = json.loads(n.read())
                url = novel['siteHostUrl']
                if url not in urls:
                    print(fp)

def removeDup(old):
    ret = list()
    urls = set()
    for each in old:
        if each['siteHostUrl'] in urls:
            continue
        else:
            urls.add(each['siteHostUrl'])
            ret.append(each)
    return ret

def countWas():
    with open('./build/all.was') as f:
        x = json.loads(f.read())
        print('all count :' , len(x))
    with open('./build/builtin.was') as f:
        y = json.loads(f.read())
        print('builtin count :' , len(y))


def add_top_100_of_yuedu():
    # https://e-c.coding.net/p/yicheng/d/YD/git/raw/master/sy.json
    with open('./yuedu.json', 'r', encoding='utf-8') as f:
        all = json.loads(f.read())
        return all[:100]

not_main = False 
if __name__ == '__main__':
    if not_main:
        checkDup()
    else:
        with open('./build/comic.was', 'w') as ff:
            comic = addAll('./comic', False)
            ff.write(json.dumps(comic))
        with open('./build/novel.json', 'w') as f:
            novel = add_top_100_of_yuedu()
            f.write(json.dumps(novel))

        
        # with open('./build/all.was', 'w') as ff:
        #     ff.write(json.dumps(addAll('./all', True), indent=4))
        # with open('./build/builtin.was','w') as ff:
        #     ff.write(json.dumps(addAll('./all/builtin', False),indent=4))
        # with open('./build/addons.was','w') as ff:
        #     ff.write(json.dumps(addAll('./all/addons'),indent=4))
        # countWas()

        # https://e-c.coding.net/p/yicheng/d/YD/git/raw/master/sy.json

        shutil.copyfile('./build/comic.was','C:/Users/forti/StudioProjects/qilin-read/app/src/main/assets/comic.was')
        shutil.copyfile('./build/novel.json','C:/Users/forti/StudioProjects/qilin-read/app/src/main/assets/novel.json')



